--- /dev/null
+From: IME Packaging Team <pkg-ime-devel@lists.alioth.debian.org>
+Date: Mon, 9 Oct 2017 22:02:39 +0800
+Subject: use-cmake-install-libdir
+
+===================================================================
+---
+ CMakeLists.txt | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 08d186a..efb51ae 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -74,7 +74,7 @@ set (DIR_PREFIX ${CMAKE_INSTALL_PREFIX})
+ set (DIR_INCLUDE ${DIR_PREFIX}/include/)
+ set (DIR_SHARE ${DIR_PREFIX}/share/)
+ set (DIR_ETC ${DIR_PREFIX}/etc/)
+-set (DIR_LIBRARY ${DIR_PREFIX}/lib${LIB_SUFFIX}/)
++set (DIR_LIBRARY ${CMAKE_INSTALL_LIBDIR})
+
+ if (DEFINED SHARE_INSTALL_PREFIX)
+ set (DIR_SHARE ${SHARE_INSTALL_PREFIX})
--- /dev/null
+From: Boyuan Yang <byang@debian.org>
+Date: Sat, 23 May 2020 11:11:46 -0400
+Subject: use-system-libraries
+
+---
+ src/CMakeLists.txt | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index 4883990..e600408 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -1,7 +1,7 @@
+ include (GenerateExportHeader)
+-include_directories(../deps/marisa-0.2.5/include)
+-include_directories(../deps/rapidjson-1.1.0)
+-include_directories(../deps/tclap-1.2.2)
++#include_directories(/usr/include/marisa)
++include_directories(/usr/include/rapidjson)
++include_directories(/usr/include/tclap)
+
+ # Library
+
+++ /dev/null
-From: Boyuan Yang <byang@debian.org>
-Date: Sat, 23 May 2020 11:11:46 -0400
-Subject: use-system-libraries
-
----
- src/CMakeLists.txt | 6 +++---
- 1 file changed, 3 insertions(+), 3 deletions(-)
-
-diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
-index 4883990..e600408 100644
---- a/src/CMakeLists.txt
-+++ b/src/CMakeLists.txt
-@@ -1,7 +1,7 @@
- include (GenerateExportHeader)
--include_directories(../deps/marisa-0.2.5/include)
--include_directories(../deps/rapidjson-1.1.0)
--include_directories(../deps/tclap-1.2.2)
-+#include_directories(/usr/include/marisa)
-+include_directories(/usr/include/rapidjson)
-+include_directories(/usr/include/tclap)
-
- # Library
-
--- /dev/null
+From: Gunnar Hjalmarsson <gunnarhj@ubuntu.com>
+Date: Sat, 6 Mar 2021 17:45:20 +0800
+Subject: Don't fetch remote images when reading docs on disk
+
+Forwarded: no
+Last-Update: 2021-01-14
+
+This fixes a privacy breach previously reported as Lintian warnings
+---
+ README.md | 8 ++++----
+ 1 file changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/README.md b/README.md
+index d941d6b..0663dcd 100644
+--- a/README.md
++++ b/README.md
+@@ -1,12 +1,12 @@
+ # Open Chinese Convert 開放中文轉換
+
+-[](https://travis-ci.org/BYVoid/OpenCC)
+-[](https://ci.appveyor.com/project/Carbo/OpenCC)
+-[](https://github.com/BYVoid/OpenCC/actions?query=workflow%3A%22Python+package%22)
++[[Travis](https://travis-ci.org/BYVoid/OpenCC)]
++[[AppVeyor](https://ci.appveyor.com/project/Carbo/OpenCC)]
++[[Python package](https://github.com/BYVoid/OpenCC/actions?query=workflow%3A%22Python+package%22)]
+
+ ## Introduction 介紹
+
+-
++OpenCC
+
+ Open Chinese Convert (OpenCC, 開放中文轉換) is an opensource project for conversions between Traditional Chinese, Simplified Chinese and Japanese Kanji (Shinjitai). It supports character-level and phrase-level conversion, character variant conversion and regional idioms among Mainland China, Taiwan and Hong Kong. This is not translation tool between Mandarin and Cantonese, etc.
+
--- /dev/null
+From: Carbo Kuo <byvoid@byvoid.com>
+Date: Thu, 25 Feb 2021 20:48:50 +0900
+Subject: Fix a bug in the calculation of DictGroup::keyMaxLength.
+
+The length should be the maximum of all sub-dictionaries in the dictionary group.
+---
+ src/DictGroup.cpp | 16 ++++++++++++++--
+ src/DictGroupTest.cpp | 32 ++++++++++++++++++++++++--------
+ 2 files changed, 38 insertions(+), 10 deletions(-)
+
+diff --git a/src/DictGroup.cpp b/src/DictGroup.cpp
+index 4ca9e33..c682e96 100644
+--- a/src/DictGroup.cpp
++++ b/src/DictGroup.cpp
+@@ -1,7 +1,7 @@
+ /*
+ * Open Chinese Convert
+ *
+- * Copyright 2010-2014 Carbo Kuo <byvoid@byvoid.com>
++ * Copyright 2010-2021 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+@@ -24,8 +24,20 @@
+
+ using namespace opencc;
+
++namespace {
++
++size_t GetKeyMaxLength(const std::list<DictPtr>& dicts) {
++ size_t keyMaxLength = 0;
++ for (const DictPtr& dict : dicts) {
++ keyMaxLength = std::max(keyMaxLength, dict->KeyMaxLength());
++ }
++ return keyMaxLength;
++}
++
++} // namespace
++
+ DictGroup::DictGroup(const std::list<DictPtr>& _dicts)
+- : keyMaxLength(0), dicts(_dicts) {}
++ : keyMaxLength(GetKeyMaxLength(_dicts)), dicts(_dicts) {}
+
+ DictGroup::~DictGroup() {}
+
+diff --git a/src/DictGroupTest.cpp b/src/DictGroupTest.cpp
+index 7003506..c91731e 100644
+--- a/src/DictGroupTest.cpp
++++ b/src/DictGroupTest.cpp
+@@ -1,7 +1,7 @@
+ /*
+ * Open Chinese Convert
+ *
+- * Copyright 2015 Carbo Kuo <byvoid@byvoid.com>
++ * Copyright 2015-2021 Carbo Kuo <byvoid@byvoid.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+@@ -25,15 +25,31 @@ protected:
+ DictGroupTest() {}
+ };
+
+-TEST_F(DictGroupTest, SimpleGroupTest) {
++TEST_F(DictGroupTest, KeyMaxLength) {
+ const DictGroupPtr& dictGroup = CreateDictGroupForConversion();
+- const auto& entry = dictGroup->Dict::MatchPrefix(utf8("Unknown"));
+- EXPECT_TRUE(entry.IsNull());
++ EXPECT_EQ(6, dictGroup->KeyMaxLength());
++ EXPECT_EQ(6, dictGroup->GetDicts().front()->KeyMaxLength());
++ EXPECT_EQ(3, dictGroup->GetDicts().back()->KeyMaxLength());
++}
+
+- const auto& matches = dictGroup->Dict::MatchAllPrefixes(utf8("干燥"));
+- EXPECT_EQ(2, matches.size());
+- EXPECT_EQ(utf8("乾燥"), matches.at(0)->GetDefault());
+- EXPECT_EQ(utf8("幹"), matches.at(1)->GetDefault());
++TEST_F(DictGroupTest, SimpleGroupTest) {
++ const DictGroupPtr& dictGroup = CreateDictGroupForConversion();
++ {
++ const auto& entry = dictGroup->Dict::MatchPrefix(utf8("Unknown"));
++ EXPECT_TRUE(entry.IsNull());
++ }
++ {
++ const auto& entry = dictGroup->Dict::MatchPrefix(utf8("里面"));
++ EXPECT_FALSE(entry.IsNull());
++ EXPECT_EQ(3, entry.Get()->KeyLength());
++ EXPECT_EQ(utf8("裏"), entry.Get()->GetDefault());
++ }
++ {
++ const auto& matches = dictGroup->Dict::MatchAllPrefixes(utf8("干燥"));
++ EXPECT_EQ(2, matches.size());
++ EXPECT_EQ(utf8("乾燥"), matches.at(0)->GetDefault());
++ EXPECT_EQ(utf8("幹"), matches.at(1)->GetDefault());
++ }
+ }
+
+ TEST_F(DictGroupTest, TaiwanPhraseGroupTest) {
--- /dev/null
+From: Carbo Kuo <byvoid@byvoid.com>
+Date: Thu, 25 Feb 2021 21:13:38 +0900
+Subject: Fix a severe performance bug in `Conversion::Convert` that caused
+ O(N^2) complexity.
+
+In `Conversion.cpp`, line 27:
+```
+ Optional<const DictEntry*> matched = dict->MatchPrefix(pstr);
+```
+pstr is a `const char*`. However, there is no overloaded function which parameter `const char*`.
+Therefore it matches `Optional<const DictEntry*> MatchPrefix(const std::string& word) const`.
+There is an implicit type conversion from `char*` to `std::string` with time complexity O(N).
+
+I added new benchmark tests. Before the fix:
+
+1: ------------------------------------------------------------------
+1: Benchmark Time CPU Iterations
+1: ------------------------------------------------------------------
+1: BM_Initialization/hk2s 1.17 ms 1.12 ms 645
+1: BM_Initialization/hk2t 0.116 ms 0.116 ms 5922
+1: BM_Initialization/jp2t 0.206 ms 0.201 ms 3500
+1: BM_Initialization/s2hk 18.2 ms 17.9 ms 40
+1: BM_Initialization/s2t 18.2 ms 18.1 ms 39
+1: BM_Initialization/s2tw 17.9 ms 17.8 ms 39
+1: BM_Initialization/s2twp 18.6 ms 18.4 ms 39
+1: BM_Initialization/t2hk 0.055 ms 0.054 ms 12907
+1: BM_Initialization/t2jp 0.120 ms 0.117 ms 5978
+1: BM_Initialization/t2s 0.988 ms 0.984 ms 710
+1: BM_Initialization/tw2s 1.08 ms 1.05 ms 672
+1: BM_Initialization/tw2sp 1.26 ms 1.24 ms 563
+1: BM_Initialization/tw2t 0.088 ms 0.083 ms 8528
+1: BM_Convert2M 413 ms 413 ms 2
+1: BM_Convert/100 1.09 ms 1.09 ms 629
+1: BM_Convert/1000 33.2 ms 33.2 ms 21
+1: BM_Convert/10000 2822 ms 2822 ms 1
+1: BM_Convert/100000 (took longer than 5 minutes, killed)
+
+Now:
+1: ------------------------------------------------------------------
+1: Benchmark Time CPU Iterations
+1: ------------------------------------------------------------------
+1: BM_Initialization/hk2s 1.07 ms 1.07 ms 650
+1: BM_Initialization/hk2t 0.114 ms 0.114 ms 6092
+1: BM_Initialization/jp2t 0.204 ms 0.200 ms 3503
+1: BM_Initialization/s2hk 18.2 ms 18.0 ms 40
+1: BM_Initialization/s2t 17.6 ms 17.6 ms 39
+1: BM_Initialization/s2tw 18.0 ms 17.9 ms 40
+1: BM_Initialization/s2twp 17.9 ms 17.9 ms 39
+1: BM_Initialization/t2hk 0.055 ms 0.055 ms 12855
+1: BM_Initialization/t2jp 0.125 ms 0.124 ms 5772
+1: BM_Initialization/t2s 1.000 ms 0.989 ms 695
+1: BM_Initialization/tw2s 1.09 ms 1.07 ms 668
+1: BM_Initialization/tw2sp 1.29 ms 1.26 ms 558
+1: BM_Initialization/tw2t 0.082 ms 0.082 ms 8558
+1: BM_Convert2M 361 ms 361 ms 2
+1: BM_Convert/100 0.741 ms 0.740 ms 948
+1: BM_Convert/1000 7.54 ms 7.52 ms 94
+1: BM_Convert/10000 76.3 ms 76.3 ms 9
+1: BM_Convert/100000 786 ms 786 ms 1
+
+This bug was reported in https://github.com/BYVoid/OpenCC/issues/478 and https://github.com/BYVoid/OpenCC/issues/517.
+---
+ src/Dict.hpp | 7 ++++
+ src/benchmark/Performance.cpp | 77 ++++++++++++++++++++++++++++++++++---------
+ 2 files changed, 69 insertions(+), 15 deletions(-)
+
+diff --git a/src/Dict.hpp b/src/Dict.hpp
+index 461d6d2..1c81034 100644
+--- a/src/Dict.hpp
++++ b/src/Dict.hpp
+@@ -49,6 +49,13 @@ public:
+ virtual Optional<const DictEntry*> MatchPrefix(const char* word,
+ size_t len) const;
+
++ /**
++ * Matches the longest matched prefix of a word.
++ */
++ Optional<const DictEntry*> MatchPrefix(const char* word) const {
++ return MatchPrefix(word, KeyMaxLength());
++ }
++
+ /**
+ * Matches the longest matched prefix of a word.
+ */
+diff --git a/src/benchmark/Performance.cpp b/src/benchmark/Performance.cpp
+index cf8d3aa..d1b6468 100644
+--- a/src/benchmark/Performance.cpp
++++ b/src/benchmark/Performance.cpp
+@@ -1,7 +1,26 @@
++/*
++ * Open Chinese Convert
++ *
++ * Copyright 2020-2021 Carbo Kuo <byvoid@byvoid.com>
++ *
++ * Licensed under the Apache License, Version 2.0 (the "License");
++ * you may not use this file except in compliance with the License.
++ * You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++
+ #include <benchmark/benchmark.h>
+ #include <fstream>
+ #include <iostream>
+ #include <memory>
++#include <sstream>
+ #include <streambuf>
+
+ #ifdef _MSC_VER
+@@ -44,21 +63,32 @@ static void BM_Initialization(benchmark::State& state,
+ state.ResumeTiming();
+ }
+ }
+-BENCHMARK_CAPTURE(BM_Initialization, hk2s, "hk2s");
+-BENCHMARK_CAPTURE(BM_Initialization, hk2t, "hk2t");
+-BENCHMARK_CAPTURE(BM_Initialization, jp2t, "jp2t");
+-BENCHMARK_CAPTURE(BM_Initialization, s2hk, "s2hk");
+-BENCHMARK_CAPTURE(BM_Initialization, s2t, "s2t");
+-BENCHMARK_CAPTURE(BM_Initialization, s2tw, "s2tw");
+-BENCHMARK_CAPTURE(BM_Initialization, s2twp, "s2twp");
+-BENCHMARK_CAPTURE(BM_Initialization, t2hk, "t2hk");
+-BENCHMARK_CAPTURE(BM_Initialization, t2jp, "t2jp");
+-BENCHMARK_CAPTURE(BM_Initialization, t2s, "t2s");
+-BENCHMARK_CAPTURE(BM_Initialization, tw2s, "tw2s");
+-BENCHMARK_CAPTURE(BM_Initialization, tw2sp, "tw2sp");
+-BENCHMARK_CAPTURE(BM_Initialization, tw2t, "tw2t");
++BENCHMARK_CAPTURE(BM_Initialization, hk2s, "hk2s")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, hk2t, "hk2t")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, jp2t, "jp2t")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, s2hk, "s2hk")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, s2t, "s2t")->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, s2tw, "s2tw")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, s2twp, "s2twp")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, t2hk, "t2hk")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, t2jp, "t2jp")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, t2s, "t2s")->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, tw2s, "tw2s")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, tw2sp, "tw2sp")
++ ->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Initialization, tw2t, "tw2t")
++ ->Unit(benchmark::kMillisecond);
+
+-static void BM_Convert(benchmark::State& state) {
++static void BM_Convert2M(benchmark::State& state) {
+ const std::string config_name = "s2t";
+ const std::string text = ReadText("zuozhuan.txt");
+ const std::unique_ptr<SimpleConverter> converter(Initialize(config_name));
+@@ -66,7 +96,24 @@ static void BM_Convert(benchmark::State& state) {
+ Convert(converter.get(), text);
+ }
+ }
+-BENCHMARK(BM_Convert)->Unit(benchmark::kMillisecond);
++BENCHMARK(BM_Convert2M)->Unit(benchmark::kMillisecond);
++
++static void BM_Convert(benchmark::State& state, int iteration) {
++ std::ostringstream os;
++ for (int i = 0; i < iteration; i++) {
++ os << "Open Chinese Convert 開放中文轉換" << i << std::endl;
++ }
++ const std::string text = os.str();
++ const std::string config_name = "s2t";
++ const std::unique_ptr<SimpleConverter> converter(Initialize(config_name));
++ for (auto _ : state) {
++ Convert(converter.get(), text);
++ }
++}
++BENCHMARK_CAPTURE(BM_Convert, 100, 100)->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Convert, 1000, 1000)->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Convert, 10000, 10000)->Unit(benchmark::kMillisecond);
++BENCHMARK_CAPTURE(BM_Convert, 100000, 100000)->Unit(benchmark::kMillisecond);
+
+ } // namespace opencc
+
+++ /dev/null
-From: Gunnar Hjalmarsson <gunnarhj@ubuntu.com>
-Date: Sat, 6 Mar 2021 17:45:20 +0800
-Subject: Don't fetch remote images when reading docs on disk
-
-Forwarded: no
-Last-Update: 2021-01-14
-
-This fixes a privacy breach previously reported as Lintian warnings
----
- README.md | 8 ++++----
- 1 file changed, 4 insertions(+), 4 deletions(-)
-
-diff --git a/README.md b/README.md
-index d941d6b..0663dcd 100644
---- a/README.md
-+++ b/README.md
-@@ -1,12 +1,12 @@
- # Open Chinese Convert 開放中文轉換
-
--[](https://travis-ci.org/BYVoid/OpenCC)
--[](https://ci.appveyor.com/project/Carbo/OpenCC)
--[](https://github.com/BYVoid/OpenCC/actions?query=workflow%3A%22Python+package%22)
-+[[Travis](https://travis-ci.org/BYVoid/OpenCC)]
-+[[AppVeyor](https://ci.appveyor.com/project/Carbo/OpenCC)]
-+[[Python package](https://github.com/BYVoid/OpenCC/actions?query=workflow%3A%22Python+package%22)]
-
- ## Introduction 介紹
-
--
-+OpenCC
-
- Open Chinese Convert (OpenCC, 開放中文轉換) is an opensource project for conversions between Traditional Chinese, Simplified Chinese and Japanese Kanji (Shinjitai). It supports character-level and phrase-level conversion, character variant conversion and regional idioms among Mainland China, Taiwan and Hong Kong. This is not translation tool between Mandarin and Cantonese, etc.
-
-use-cmake-install-libdir.patch
-0003-use-system-libraries.patch
+0001-use-cmake-install-libdir.patch
+0002-use-system-libraries.patch
0003-data-Explicitly-use-python3.patch
-no-remote-images-when-reading-docs-on-disk.patch
+0004-no-remote-images-when-reading-docs-on-disk.patch
0005-Use-system-googletest.patch
+0006-Fix-a-bug-in-the-calculation-of-DictGroup-keyMaxLeng.patch
+0007-Fix-a-severe-performance-bug-in-Conversion-Convert-t.patch
+++ /dev/null
-From: IME Packaging Team <pkg-ime-devel@lists.alioth.debian.org>
-Date: Mon, 9 Oct 2017 22:02:39 +0800
-Subject: use-cmake-install-libdir
-
-===================================================================
----
- CMakeLists.txt | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 08d186a..efb51ae 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -74,7 +74,7 @@ set (DIR_PREFIX ${CMAKE_INSTALL_PREFIX})
- set (DIR_INCLUDE ${DIR_PREFIX}/include/)
- set (DIR_SHARE ${DIR_PREFIX}/share/)
- set (DIR_ETC ${DIR_PREFIX}/etc/)
--set (DIR_LIBRARY ${DIR_PREFIX}/lib${LIB_SUFFIX}/)
-+set (DIR_LIBRARY ${CMAKE_INSTALL_LIBDIR})
-
- if (DEFINED SHARE_INSTALL_PREFIX)
- set (DIR_SHARE ${SHARE_INSTALL_PREFIX})
projects / opencc.git / commitdiff